{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SSA dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from typing import Final\n",
    "from collections.abc import Callable\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "from config import data_raw_folder, data_processed_folder\n",
    "from timeeval import Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (20, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking for source datasets in /home/projects/akita/data/benchmark-data/data-raw/SSA and\n",
      "saving processed datasets in /home/projects/akita/data/benchmark-data/data-processed\n"
     ]
    }
   ],
   "source": [
    "dataset_collection_name = \"SSA\"\n",
    "source_folder = Path(data_raw_folder) / \"SSA\"\n",
    "target_folder = data_processed_folder\n",
    "\n",
    "print(f\"Looking for source datasets in {Path(source_folder).absolute()} and\\nsaving processed datasets in {Path(target_folder).absolute()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "file handling and transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_files() -> list[str]:\n",
    "    return [f.name for f in (source_folder / \"data\").iterdir() if f.is_file()]\n",
    "\n",
    "def get_source_file(name: str, label: bool = False) -> Path:\n",
    "    if label:\n",
    "        parts = name.split(\"_\")\n",
    "        prefix = parts[0]\n",
    "        postfix = parts[2]\n",
    "        return source_folder / \"label\" / f\"{prefix}_truth_{postfix}.csv\"\n",
    "    else:\n",
    "        return source_folder / \"data\" / name\n",
    "\n",
    "def process(file: str, target: Path, is_train: bool = False) -> int:\n",
    "    df_data = pd.read_csv(get_source_file(file), names=[\"value\"])\n",
    "    df_label = pd.read_csv(get_source_file(file, label=True), names=[\"is_anomaly\"])\n",
    "    df = pd.merge(df_data, df_label, left_index=True, right_index=True)\n",
    "    df.index.name = \"timestamp\"\n",
    "    df.to_csv(target, index=True)\n",
    "    return len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Directories /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA already exist\n"
     ]
    }
   ],
   "source": [
    "# shared by all datasets\n",
    "dataset_type = \"real\"\n",
    "train_is_normal = False\n",
    "train_type = \"unsupervised\"\n",
    "input_type = \"univariate\"\n",
    "datetime_index = False\n",
    "\n",
    "# create target directory\n",
    "dataset_subfolder = Path(input_type) / dataset_collection_name\n",
    "target_subfolder = target_folder / dataset_subfolder\n",
    "try:\n",
    "    os.makedirs(target_subfolder)\n",
    "    print(f\"Created directories {target_subfolder}\")\n",
    "except FileExistsError:\n",
    "    print(f\"Directories {target_subfolder} already exist\")\n",
    "    pass\n",
    "\n",
    "dm = Datasets(target_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed source dataset stb_data_15 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-15.test.csv\n",
      "Processed source dataset stb_data_9 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-9.test.csv\n",
      "Processed source dataset stb_data_32 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-32.test.csv\n",
      "Processed source dataset stb_data_10 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-10.test.csv\n",
      "Processed source dataset stb_data_5 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-5.test.csv\n",
      "Processed source dataset stb_data_20 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-20.test.csv\n",
      "Processed source dataset stb_data_19 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-19.test.csv\n",
      "Processed source dataset stb_data_3 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-3.test.csv\n",
      "Processed source dataset stb_data_2 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-2.test.csv\n",
      "Processed source dataset stb_data_12 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-12.test.csv\n",
      "Processed source dataset stb_data_6 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-6.test.csv\n",
      "Processed source dataset stb_data_31 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-31.test.csv\n",
      "Processed source dataset stb_data_14 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-14.test.csv\n",
      "Processed source dataset stb_data_8 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-8.test.csv\n",
      "Processed source dataset stb_data_11 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-11.test.csv\n",
      "Processed source dataset stb_data_4 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-4.test.csv\n",
      "Processed source dataset stb_data_7 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-7.test.csv\n",
      "Processed source dataset stb_data_28 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-28.test.csv\n",
      "Processed source dataset stb_data_17 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-17.test.csv\n",
      "Processed source dataset stb_data_13 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-13.test.csv\n",
      "Processed source dataset stb_data_25 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-25.test.csv\n",
      "Processed source dataset stb_data_29 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-29.test.csv\n",
      "Processed source dataset stb_data_18 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/SSA/stb-18.test.csv\n"
     ]
    }
   ],
   "source": [
    "for f in list_files():\n",
    "    parts = f.split(\"_\")\n",
    "    dataset_name = f\"{parts[0]}-{parts[2]}\"\n",
    "    filename = f\"{dataset_name}.test.csv\"\n",
    "    path = dataset_subfolder / filename\n",
    "    target_filepath = target_subfolder / filename\n",
    "\n",
    "    dataset_length = process(f, target_filepath)\n",
    "\n",
    "    # save metadata\n",
    "    dm.add_dataset((dataset_collection_name, dataset_name),\n",
    "        train_path = None,\n",
    "        test_path = path,\n",
    "        dataset_type = dataset_type,\n",
    "        datetime_index = datetime_index,\n",
    "        split_at = None,\n",
    "        train_type = train_type,\n",
    "        train_is_normal = train_is_normal,\n",
    "        input_type = input_type,\n",
    "        dataset_length = dataset_length\n",
    "    )\n",
    "    print(f\"Processed source dataset {f} -> {target_filepath}\")\n",
    "\n",
    "# save metadata of benchmark\n",
    "dm.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>train_path</th>\n",
       "      <th>test_path</th>\n",
       "      <th>dataset_type</th>\n",
       "      <th>datetime_index</th>\n",
       "      <th>split_at</th>\n",
       "      <th>train_type</th>\n",
       "      <th>train_is_normal</th>\n",
       "      <th>input_type</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>collection_name</th>\n",
       "      <th>dataset_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"23\" valign=\"top\">SSA</th>\n",
       "      <th>stb-10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-10.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-11.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-12.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-13.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-14.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-15.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>30493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-17.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>25296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-18.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>25283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-19.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>30493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-2.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-20</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-20.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-25</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-25.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>20734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-28</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-28.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>20734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-29</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-29.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>20734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-3.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>30356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-31</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-31.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>20734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-32</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-32.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>20733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-4.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>30356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-5.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-6.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>25818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-7.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>24906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-8.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stb-9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/SSA/stb-9.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>29517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             train_path                       test_path  \\\n",
       "collection_name dataset_name                                              \n",
       "SSA             stb-10              NaN  univariate/SSA/stb-10.test.csv   \n",
       "                stb-11              NaN  univariate/SSA/stb-11.test.csv   \n",
       "                stb-12              NaN  univariate/SSA/stb-12.test.csv   \n",
       "                stb-13              NaN  univariate/SSA/stb-13.test.csv   \n",
       "                stb-14              NaN  univariate/SSA/stb-14.test.csv   \n",
       "                stb-15              NaN  univariate/SSA/stb-15.test.csv   \n",
       "                stb-17              NaN  univariate/SSA/stb-17.test.csv   \n",
       "                stb-18              NaN  univariate/SSA/stb-18.test.csv   \n",
       "                stb-19              NaN  univariate/SSA/stb-19.test.csv   \n",
       "                stb-2               NaN   univariate/SSA/stb-2.test.csv   \n",
       "                stb-20              NaN  univariate/SSA/stb-20.test.csv   \n",
       "                stb-25              NaN  univariate/SSA/stb-25.test.csv   \n",
       "                stb-28              NaN  univariate/SSA/stb-28.test.csv   \n",
       "                stb-29              NaN  univariate/SSA/stb-29.test.csv   \n",
       "                stb-3               NaN   univariate/SSA/stb-3.test.csv   \n",
       "                stb-31              NaN  univariate/SSA/stb-31.test.csv   \n",
       "                stb-32              NaN  univariate/SSA/stb-32.test.csv   \n",
       "                stb-4               NaN   univariate/SSA/stb-4.test.csv   \n",
       "                stb-5               NaN   univariate/SSA/stb-5.test.csv   \n",
       "                stb-6               NaN   univariate/SSA/stb-6.test.csv   \n",
       "                stb-7               NaN   univariate/SSA/stb-7.test.csv   \n",
       "                stb-8               NaN   univariate/SSA/stb-8.test.csv   \n",
       "                stb-9               NaN   univariate/SSA/stb-9.test.csv   \n",
       "\n",
       "                             dataset_type  datetime_index  split_at  \\\n",
       "collection_name dataset_name                                          \n",
       "SSA             stb-10               real           False       NaN   \n",
       "                stb-11               real           False       NaN   \n",
       "                stb-12               real           False       NaN   \n",
       "                stb-13               real           False       NaN   \n",
       "                stb-14               real           False       NaN   \n",
       "                stb-15               real           False       NaN   \n",
       "                stb-17               real           False       NaN   \n",
       "                stb-18               real           False       NaN   \n",
       "                stb-19               real           False       NaN   \n",
       "                stb-2                real           False       NaN   \n",
       "                stb-20               real           False       NaN   \n",
       "                stb-25               real           False       NaN   \n",
       "                stb-28               real           False       NaN   \n",
       "                stb-29               real           False       NaN   \n",
       "                stb-3                real           False       NaN   \n",
       "                stb-31               real           False       NaN   \n",
       "                stb-32               real           False       NaN   \n",
       "                stb-4                real           False       NaN   \n",
       "                stb-5                real           False       NaN   \n",
       "                stb-6                real           False       NaN   \n",
       "                stb-7                real           False       NaN   \n",
       "                stb-8                real           False       NaN   \n",
       "                stb-9                real           False       NaN   \n",
       "\n",
       "                                train_type  train_is_normal  input_type  \\\n",
       "collection_name dataset_name                                              \n",
       "SSA             stb-10        unsupervised            False  univariate   \n",
       "                stb-11        unsupervised            False  univariate   \n",
       "                stb-12        unsupervised            False  univariate   \n",
       "                stb-13        unsupervised            False  univariate   \n",
       "                stb-14        unsupervised            False  univariate   \n",
       "                stb-15        unsupervised            False  univariate   \n",
       "                stb-17        unsupervised            False  univariate   \n",
       "                stb-18        unsupervised            False  univariate   \n",
       "                stb-19        unsupervised            False  univariate   \n",
       "                stb-2         unsupervised            False  univariate   \n",
       "                stb-20        unsupervised            False  univariate   \n",
       "                stb-25        unsupervised            False  univariate   \n",
       "                stb-28        unsupervised            False  univariate   \n",
       "                stb-29        unsupervised            False  univariate   \n",
       "                stb-3         unsupervised            False  univariate   \n",
       "                stb-31        unsupervised            False  univariate   \n",
       "                stb-32        unsupervised            False  univariate   \n",
       "                stb-4         unsupervised            False  univariate   \n",
       "                stb-5         unsupervised            False  univariate   \n",
       "                stb-6         unsupervised            False  univariate   \n",
       "                stb-7         unsupervised            False  univariate   \n",
       "                stb-8         unsupervised            False  univariate   \n",
       "                stb-9         unsupervised            False  univariate   \n",
       "\n",
       "                              length  \n",
       "collection_name dataset_name          \n",
       "SSA             stb-10         29518  \n",
       "                stb-11         29517  \n",
       "                stb-12         29517  \n",
       "                stb-13         29517  \n",
       "                stb-14         29518  \n",
       "                stb-15         30493  \n",
       "                stb-17         25296  \n",
       "                stb-18         25283  \n",
       "                stb-19         30493  \n",
       "                stb-2          29553  \n",
       "                stb-20         29518  \n",
       "                stb-25         20734  \n",
       "                stb-28         20734  \n",
       "                stb-29         20734  \n",
       "                stb-3          30356  \n",
       "                stb-31         20734  \n",
       "                stb-32         20733  \n",
       "                stb-4          30356  \n",
       "                stb-5          29520  \n",
       "                stb-6          25818  \n",
       "                stb-7          24906  \n",
       "                stb-8          29519  \n",
       "                stb-9          29517  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dm.refresh()\n",
    "dm.df().loc[(slice(dataset_collection_name,dataset_collection_name), slice(None))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experimentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = list_files()\n",
    "f = files[0]\n",
    "print(\"data\", get_source_file(f))\n",
    "print(\"label\", get_source_file(f, label=True))\n",
    "df_data = pd.read_csv(get_source_file(f), names=[\"value\"])\n",
    "df_label = pd.read_csv(get_source_file(f, label=True), names=[\"is_anomaly\"])\n",
    "df = pd.merge(df_data, df_label, left_index=True, right_index=True)\n",
    "df.index.name = \"timestamp\"\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "timeeval",
   "language": "python",
   "name": "timeeval"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
